package com.example.demo;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

public class GithubRepoPageProcessor implements PageProcessor {


    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
    List<Object> list = new ArrayList<>();
    List<List<Object>> csvList = new ArrayList<>();
    static String  url = "https://pub.dev/flutter/packages?platform=android+ios&sort=like";
    volatile boolean flag=true;
    @Override
    public void process(Page page) {
        System.out.println(page.getHtml());
      //  System.out.println(page.getHtml().xpath("//span[@class='packages-score-value-number']/tidyText()").toString());
      //  page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/[\\w\\-]+/[\\w\\-]+)").all());
     //   page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/[\\w\\-])").all());
//        page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());
//        page.putField("name", page.getHtml().xpath("//h1[@class='entry-title public']/strong/a/text()").toString());
//        if (page.getResultItems().get("name")==null){
//            //skip this page
//            page.setSkip(true);
//        }
//        page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"));
        int m = 0;

        for (int i = 1; i <= 100&&flag; i++) {
            m=i;
          //获取下一页的链接，将当前页数拼接到url上
            String nextUrl=url+"&page="+i;
           //将下一页链接添加到爬虫队列中
            page.addTargetRequest(nextUrl);
 //        page.putField("label", page.getHtml().xpath("//h3[@class='packages-title']/a/text()").all());
//        page.putField("health", page.getHtml().xpath("//div[@class='packages-score packages-score-health']//span[@class='packages-score-value-number']/text()").all());
//        page.putField("popularity", page.getHtml().xpath("//div[@class='packages-score packages-score-popularity']//span[@class='packages-score-value-number']/text()").all());

            List<String>titles = page.getHtml().xpath("//h3[@class='packages-title']/a/text()").all(); //获取列表的条数
            if(titles.size()<10){
                flag=false;
              break;
            }
        }

        List<String>titles = page.getHtml().xpath("//h3[@class='packages-title']/a/text()").all(); //获取列表的条数
        List<String> popularitys =  page.getHtml().xpath("//div[@class='packages-score packages-score-popularity']//span[@class='packages-score-value-number']/text()").all();
        List<String> healths =page.getHtml().xpath("//div[@class='packages-score packages-score-health']//span[@class='packages-score-value-number']/text()").all();
        List<String> likes =page.getHtml().xpath("//div[@class='packages-score packages-score-like']//span[@class='packages-score-value-number']/text()").all();
        for (int j = 0; j < titles.size(); j++) {
            PubData pubData =  new PubData();
            pubData.setLabel(titles.get(j));
            List<Object>deils = new ArrayList<>();
            deils.add(titles.get(j));
            deils.add(likes.get(j));
            deils.add(healths.get(j));
            deils.add(popularitys.get(j));
            csvList.add(deils);
            pubData.setPopularity(popularitys.get(j));
            pubData.setHealth(healths.get(j));
//            list.add(pubData);
//            pubData.setLabel(page.getHtml().xpath("//h3[@class='packages-title']/a/text()").get());
//            pubData.setPopularity(page.getHtml().xpath("//div[@class='packages-score packages-score-popularity']//span[@class='packages-score-value-number']/text()").get());
//            pubData.setHealth(page.getHtml().xpath("//div[@class='packages-score packages-score-health']//span[@class='packages-score-value-number']/text()").get());

            list.add(pubData);
        }
        System.out.println(list.size());
        if(!flag){
            page.putField("privateRecommend",list);
        }
        if(m>99){
            List<Object> head = new ArrayList<>();
            head.add("title");
            head.add("likes");
            head.add("Health");
            head.add("Popularity");;
            createCSVFile(head,csvList,"D://","aaa");
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
            Spider.create(new GithubRepoPageProcessor())
                    .addUrl(url+"&page=1")
                    .addPipeline(new JsonFilePipeline("D:\\webmagic\\"))
                    .thread(1).run();
    }
    public static File createCSVFile(List<Object> head, List<List<Object>> dataList,String outPutPath, String filename) {
        File csvFile = null;
        BufferedWriter csvWtriter = null;
        try {
            csvFile = new File(outPutPath + File.separator + filename + ".csv");
            File parent = csvFile.getParentFile();
            if (parent != null && !parent.exists()) {
                parent.mkdirs();
            }
            csvFile.createNewFile();

            // GB2312使正确读取分隔符","
            csvWtriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
                    csvFile), "GB2312"), 1024);
            // 写入文件头部
            writeRow(head, csvWtriter);

            // 写入文件内容
            for (List<Object> row : dataList) {
                writeRow(row, csvWtriter);
            }
            csvWtriter.flush();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                csvWtriter.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return csvFile;
    }

    /**
     * 写一行数据方法
     * @param row
     * @param csvWriter
     * @throws IOException
     */
    private static void writeRow(List<Object> row, BufferedWriter csvWriter) throws IOException {
        // 写入文件头部
        for (Object data : row) {
            StringBuffer sb = new StringBuffer();
            String rowStr = sb.append("\"").append(data).append("\",").toString();
            csvWriter.write(rowStr);
        }
        csvWriter.newLine();
    }

}